package com.bestv.search.spider.douban.sync;

import com.bestv.search.spider.douban.model.DouBanReview;
import com.bestv.search.spider.douban.model.DouBanReviewsBaseInfo;
import com.bestv.search.spider.douban.model.DouBanVideoBaseInfo;
import com.bestv.search.spider.douban.service.DouBanVideoBaseInfoManager;
import com.bestv.search.spider.douban.util.DouBanAppKeyPool;
import com.bestv.search.spider.douban.util.DouBanUtil;
import com.bestv.search.spider.util.SpiderConst;
import com.bestv.search.spider.util.SpiderUtil;
import com.bestv.search.spider.util.StringUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.Div;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.springframework.scheduling.quartz.QuartzJobBean;

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Map.Entry;

public class DouBanVideoReviewsSyncer extends QuartzJobBean {
    private static boolean isRunning = false;
    private final Log log = LogFactory.getLog(DouBanVideoReviewsSyncer.class);
    private DouBanVideoBaseInfoManager douBanVideoBaseInfoManager;
    private String infoFolder;

    @Override
    protected void executeInternal(JobExecutionContext arg0) throws JobExecutionException {
        if (!isRunning) {
            isRunning = true;

            Calendar c = Calendar.getInstance();
            String fileFolderPath = getReviewsFileFolderByDate(c.getTime());

            // remove over 7 days' files.
            String parentFolder = fileFolderPath + File.separator + SpiderConst.POINT + SpiderConst.POINT
                    + File.separator;
            String[] overdueFolders = SpiderUtil.getOver7DaysFolders(parentFolder);
            if (overdueFolders != null) {
                for (String path : overdueFolders) {
                    SpiderUtil.deleteDirectory(parentFolder + path);
                }
            }

            String isRunningFlagFile = fileFolderPath + File.separator + SpiderConst.ISRUNNING_FLAG_FILE;
            SpiderUtil.createFile(isRunningFlagFile);

            log.info("Start sync reviews from douban @ " + c.getTime());
            String file = fileFolderPath + File.separator + SpiderConst.DOUBAN_ALL_VIDEOS_REVIEWS_FILE;

            boolean syncFinished = false;
            do {
                if (!DouBanAppKeyPool.isBusy) {
                    // Gets Info videos that will be used to sync reviews from
                    // douban
                    List<DouBanVideoBaseInfo> list = douBanVideoBaseInfoManager.getValidInfoBySyncReviewsTime(c
                            .getTime());

                    // Sync reviews by DouBanVideoBaseInfo list
                    syncFinished = syncVideoReviews(list, file, c.getTime());
                } else {
                    try {
                        Thread.sleep(SpiderConst.THREAD_CHECK_INTERVAL_SECOND * 1000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            } while (!syncFinished);
            SpiderUtil.deleteFile(isRunningFlagFile);

            isRunning = false;
            log.info("End sync reviews from douban @" + new Date());
        } else {
            log.info("Last reviews sync process is still running");
        }
    }

    /**
     * Sync reviews
     *
     * @param list
     * @param file
     * @param date
     * @return Whether sync finished.
     */
    public boolean syncVideoReviews(List<DouBanVideoBaseInfo> list, String file, Date date) {
        DouBanUtil util = new DouBanUtil();
        if (list == null || list.size() == 0) {
            // Reviews has sync end.
            return true;
        } else {
            for (DouBanVideoBaseInfo info : list) {
                // Subject_id is video id of douban, so it must be not null or
                // empty
                if (info.getSubject_id() != null && !info.getSubject_id().equals(SpiderConst.EMPTY)) {
                    // Build reviews general info url by subject_id
                    StringBuffer url = StringUtil.buildBuffer(new String[]{SpiderConst.DOUBAN_SUBJECT_URL,
                            info.getSubject_id(), SpiderConst.BACK_SLANT, SpiderConst.REVIEWS});

                    // Get response by url
                    String response = util.sendRequest(url.toString());
                    if (response != null && !response.equals(SpiderConst.EMPTY)) {

                        // Handle response to generate DouBanReviewsBaseInfo
                        DouBanReviewsBaseInfo baseInfo = getReviewsBaseInfo(response, info.getId());
                        if (baseInfo.getTotalCount() > 0) {
                            // Sort for reviews level, DESC
                            Collections.sort(baseInfo.getLevelInfo(), new Comparator<Entry<Integer, Integer>>() {
                                @Override
                                public int compare(Entry<Integer, Integer> baseInfo1, Entry<Integer, Integer> baseInfo2) {
                                    return baseInfo2.getKey() - baseInfo1.getKey();
                                }
                            });

                            // gets reviews general info that order by level
                            // DESC
                            int syncReviewsCount = 0;
                            List<DouBanReview> reviews = new ArrayList<DouBanReview>();
                            List<Entry<Integer, Integer>> levels = baseInfo.getLevelInfo();
                            for (int i = 0; i < levels.size(); i++) {
                                Entry<Integer, Integer> entry = levels.get(i);
                                int level = entry.getKey();
                                int count = entry.getValue();
                                int start = 0;

                                // Gets reviews summary info list.
                                int times = 0;
                                do {
                                    StringBuffer reviewsByScopeUrl = StringUtil.buildBuffer(new String[]{
                                            url.toString(), SpiderConst.QUESTION_MARK, SpiderConst.SCORE,
                                            SpiderConst.EQUAL_FLAG, String.valueOf(level), SpiderConst.URL_SPLIT_FLAG,
                                            SpiderConst.START, SpiderConst.EQUAL_FLAG, String.valueOf(start),
                                            SpiderConst.URL_SPLIT_FLAG, SpiderConst.LIMIT, SpiderConst.EQUAL_FLAG,
                                            String.valueOf(SpiderConst.DOUBAN_REVIEWS_PAGE_COUNT)});
                                    String reviewsListResponse = util.sendRequest(reviewsByScopeUrl.toString());

                                    // adds this times' reviews summary info
                                    // list to total info list
                                    if (reviewsListResponse != null) {
                                        List<DouBanReview> reviewsList = getReviewsList(reviewsListResponse,
                                                info.getId());
                                        reviews.addAll(reviewsList);
                                        syncReviewsCount += reviewsList.size();
                                        start += SpiderConst.DOUBAN_REVIEWS_PAGE_COUNT;
                                    }

                                    // Max re-try 10 times.
                                    times++;
                                    if (times == 5) {
                                        // Sleep 3 seconds to don't make it
                                        // access too often
                                        try {
                                            Thread.sleep(3 * 1000);
                                        } catch (InterruptedException e) {
                                            e.printStackTrace();
                                        }
                                    } else if (times > 10) {
                                        break;
                                    }
                                } while (start < count && syncReviewsCount < SpiderConst.DOUBAN_REVIEWS_SYNC_LIMIT);

                                // Gets enough reviews, break this loop
                                if (syncReviewsCount >= SpiderConst.DOUBAN_REVIEWS_SYNC_LIMIT) {
                                    break;
                                }
                            }

                            if (reviews != null && reviews.size() > 0) {
                                // Sort for reviews by totalNum, DESC
                                Collections.sort(reviews, new Comparator<DouBanReview>() {
                                    @Override
                                    public int compare(DouBanReview review1, DouBanReview review2) {
                                        return review2.getTotalNum() - review1.getTotalNum();
                                    }
                                });

                                // Gets review text of top
                                // SpiderConst.DOUBAN_REVIEWS_USEFUL_LIMIT
                                int top = Math.min(SpiderConst.DOUBAN_REVIEWS_USEFUL_LIMIT, reviews.size());
                                List<DouBanReview> topReviews = new ArrayList<DouBanReview>();
                                for (int i = 0; i < top; i++) {
                                    // Gets reviews details message.
                                    StringBuffer reviewUrl = StringUtil.buildBuffer(new String[]{
                                            SpiderConst.DOUBAN_REVIEW_URL, reviews.get(i).getId()});
                                    String reviewResponse = util.sendRequest(reviewUrl.toString());

                                    if (reviewResponse != null && !reviewResponse.equals(SpiderConst.EMPTY)) {
                                        DouBanReview currReivew = reviews.get(i);
                                        currReivew = getReviewDetails(currReivew, reviewResponse, info.getId());
                                        topReviews.add(currReivew);
                                    }

                                    // Sleep 2 seconds to don't make it access
                                    // too often
                                    try {
                                        Thread.sleep(3 * 1000);
                                    } catch (InterruptedException e) {
                                        e.printStackTrace();
                                    }
                                }

                                if (topReviews.size() > 0) {
                                    // add reviews to file
                                    DouBanUtil.addTextToFile(convertToJsonText(info, topReviews), file);
                                }
                            }

                            // Update review sync time and reviews count
                            info.setReviewsSyncTime(date);
                            info.setReviewsCount(baseInfo.getTotalCount());
                            // Update info to DB
                            douBanVideoBaseInfoManager.update(info);
                        } else {
                            info.setReviewsSyncTime(date);
                            // Update info to DB
                            douBanVideoBaseInfoManager.update(info);
                        }
                    } else {
                        info.setReviewsSyncTime(date);
                        // Update info to DB
                        douBanVideoBaseInfoManager.update(info);
                    }
                } else {
                    info.setReviewsSyncTime(date);
                    // Update info to DB
                    douBanVideoBaseInfoManager.update(info);
                }

                // Sleep 3 seconds to don't make it access too often
                try {
                    Thread.sleep(3 * 1000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
            return false;
        }
    }

    // Gets review details description from HTML response
    private DouBanReview getReviewDetails(DouBanReview reivew, String response, String infoId) {
        Parser p = new Parser();
        try {
            p.setInputHTML(response);

            // gets main-hd div
            NodeFilter contentDivFilter = new HasAttributeFilter(SpiderConst.ID, SpiderConst.CONTENT);
            NodeList contentNodes = p.parse(contentDivFilter);
            // get first one, due to get DIV by id.
            Node contentDiv = contentNodes.elementAt(0);

            // gets author
            NodeFilter reviewerFilter = new HasAttributeFilter(SpiderConst.PROPERTY, SpiderConst.V_REVIEWER);
            NodeList reviewerDivs = new NodeList();
            contentDiv.collectInto(reviewerDivs, reviewerFilter);
            reivew.setAuthor(reviewerDivs.elementAt(0).toPlainTextString());

            // gets published time
            NodeFilter publishedTimeFilter = new HasAttributeFilter(SpiderConst.PROPERTY, SpiderConst.V_DTREVIEWED);
            NodeList timeDivs = new NodeList();
            contentDiv.collectInto(timeDivs, publishedTimeFilter);
            reivew.setPublished(timeDivs.elementAt(0).toPlainTextString());

            // gets review description detail
            NodeFilter descriptionFilter = new HasAttributeFilter(SpiderConst.PROPERTY, SpiderConst.V_DESCRIPTION);
            NodeList descriptionDivs = new NodeList();
            contentDiv.collectInto(descriptionDivs, descriptionFilter);
            // Sets details text
            reivew.setText(descriptionDivs.elementAt(0).toPlainTextString());

            return reivew;
        } catch (ParserException e) {
            log.error("Get review detail description failed for info id " + infoId);
            log.error(e);
            e.printStackTrace();
        }
        return null;
    }

    // Gets review list from HTML response
    private List<DouBanReview> getReviewsList(String response, String infoId) {
        List<DouBanReview> result = new ArrayList<DouBanReview>();
        Parser p = new Parser();
        try {
            p.setInputHTML(response);

            // gets all review-short div
            NodeFilter divFilter = new HasAttributeFilter(SpiderConst.CLASS, SpiderConst.REVIEW_SHORT_CLS);
            NodeList nodes = p.parse(divFilter);

            // Gets short review
            for (int i = 0; i < nodes.size(); i++) {
                Node node = nodes.elementAt(i);
                if (node instanceof Div) {
                    Div div = (Div) node;
                    String divId = div.getAttribute(SpiderConst.ID);
                    if (divId.matches(SpiderConst.REVIEW_SHORT_ID_FORMAT)) {
                        DouBanReview reivew = new DouBanReview();
                        String[] ids = divId.split(SpiderConst.UNDER_LINE);
                        reivew.setId(ids[1]);

                        NodeFilter spanFilter = new TagNameFilter(SpiderConst.SPAN);
                        NodeList spanTags = new NodeList();
                        node.collectInto(spanTags, spanFilter);

                        String[] usefulInfos = spanTags.elementAt(1).toPlainTextString().split(SpiderConst.BACK_SLANT);
                        reivew.setUsefullNum(Integer.valueOf(usefulInfos[0]));
                        reivew.setTotalNum(Integer.valueOf(usefulInfos[1]));

                        result.add(reivew);
                    } else {
                        log.error("Short review id doesn't match REVIEW_SHORT_ID_FORMAT, please check div id " + divId);
                    }
                } else {
                    log.error("Get short review list DIV failed for info id" + infoId);
                }
            }
        } catch (ParserException e) {
            log.error("Get short review list failed for inof id " + infoId);
            log.error(e);
            e.printStackTrace();
        }
        return result;
    }

    // Gets review base info from HTML response
    private DouBanReviewsBaseInfo getReviewsBaseInfo(String response, String infoId) {
        DouBanReviewsBaseInfo result = new DouBanReviewsBaseInfo();
        Parser p = new Parser();
        try {
            p.setInputHTML(response);

            // Gets all reviews base info DIV, etc, 5星 - 8， 4星 - 15 and so on
            NodeFilter divFilter = new HasAttributeFilter(SpiderConst.CLASS, SpiderConst.RATING_LIST_CLS);
            NodeList nodes = p.parse(divFilter);

            // Just get the first one node, due to reviews base info DIV just
            // have one.
            NodeFilter spanFilter = new TagNameFilter(SpiderConst.SPAN);
            NodeList spanTags = new NodeList();
            nodes.elementAt(0).collectInto(spanTags, spanFilter);
            for (int i = 0; i < spanTags.size(); i++) {
                Node reviewBaseInfoSpan = spanTags.elementAt(i);
                String reviewBaseInfo = reviewBaseInfoSpan.toPlainTextString();
                if (reviewBaseInfo != null && !reviewBaseInfo.equals(SpiderConst.EMPTY)) {
                    String[] array = reviewBaseInfo.split(SpiderConst.LINE);
                    if (array[0].trim().startsWith(SpiderConst.LEVEL_ALL_CHINESE)) {
                        result.setTotalCount(Integer.valueOf(array[1].trim()));
                    } else {
                        result.getLevelInfo().add(
                                new AbstractMap.SimpleEntry<Integer, Integer>(convertToLevel(array[0].trim()), Integer
                                        .valueOf(array[1].trim())));
                    }
                }
            }
        } catch (ParserException e) {
            log.error("Gets reivew base info failed for info id " + infoId);
            log.error(e);
            e.printStackTrace();
        }
        return result;
    }

    private String getReviewsFileFolderByDate(Date date) {
        String reviewsFolder = this.getInfoFolder() + File.separator + SpiderConst.REVIEWS;
        File folder = new File(reviewsFolder);
        if (!folder.exists()) {
            folder.mkdir();
        }

        reviewsFolder = reviewsFolder + File.separator + SpiderConst.DOUBAN;
        folder = new File(reviewsFolder);
        if (!folder.exists()) {
            folder.mkdir();
        }

        SimpleDateFormat format = new SimpleDateFormat(SpiderConst.DATETIME_FORMAT);
        String dateTimeFolder = reviewsFolder + File.separator + format.format(date);
        folder = new File(dateTimeFolder);
        if (!folder.exists()) {
            folder.mkdir();
        }
        return dateTimeFolder;
    }

    private String convertToJsonText(DouBanVideoBaseInfo info, List<DouBanReview> reviews) {
        JSONObject json = new JSONObject();
        json.put(SpiderConst.ID, info.getId());
        json.put(SpiderConst.NAME, info.getName());

        JSONArray array = new JSONArray();
        for (DouBanReview review : reviews) {
            JSONObject reviewJson = new JSONObject();
            reviewJson.put(SpiderConst.ID, review.getId());
            reviewJson.put(SpiderConst.AUTHOR, review.getAuthor());
            reviewJson.put(SpiderConst.TOTAL, review.getTotalNum());
            reviewJson.put(SpiderConst.USEFUL, review.getUsefullNum());
            reviewJson.put(SpiderConst.TEXT, review.getText());
            reviewJson.put(SpiderConst.PUBLISHED, review.getPublished());
            array.add(reviewJson);
        }
        json.put(SpiderConst.REVIEWS, array.toString());

        return json.toString();
    }

    private int convertToLevel(String levelName) {
        if (levelName == null || levelName.equals(SpiderConst.EMPTY)) {
            return 0;
        } else if (levelName.equals(SpiderConst.LEVEL_1)) {
            return 1;
        } else if (levelName.equals(SpiderConst.LEVEL_2)) {
            return 2;
        } else if (levelName.equals(SpiderConst.LEVEL_3)) {
            return 3;
        } else if (levelName.equals(SpiderConst.LEVEL_4)) {
            return 4;
        } else if (levelName.equals(SpiderConst.LEVEL_5)) {
            return 5;
        } else {
            return 0;
        }
    }

    public DouBanVideoBaseInfoManager getDouBanVideoBaseInfoManager() {
        return douBanVideoBaseInfoManager;
    }

    public void setDouBanVideoBaseInfoManager(DouBanVideoBaseInfoManager douBanVideoBaseInfoManager) {
        this.douBanVideoBaseInfoManager = douBanVideoBaseInfoManager;
    }

    public String getInfoFolder() {
        return infoFolder;
    }

    public void setInfoFolder(String infoFolder) {
        this.infoFolder = infoFolder;
    }
}